#!/usr/bin/env python

#################################################################################
#
# Copyright (c) 2006 Michigan State University
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#################################################################################

#################################################################################
#
#	Author:
#		Nathan Collins <npcollins@gmail.com>
#
#################################################################################

from querymgr import *
from oaiparse import *
from utility import *
import re
from time import sleep
from threading import Thread
import sys, os
import language
from oailog import hErr, logDone
from urlcheck import *
from oaivars import *


def read_reposit(filename = 'reposit'):
	# get list of repositories from file
	reposit = []
	fh = open(filename,'r')
	for line in fh:
		reposit.append(line.strip())
	fh.close()
	return reposit

repositories = read_reposit()

# give the class a repository uri and it will harvest the records into the db
class repoHarv(Thread):
	def __init__(self, serverURI, quiet=0):
		if quiet == 1:
			Thread.__init__(self)
		self.serverURI = serverURI
		
		# am I done?
		self.finished = 0

		# did I complete successfully?
		self.complete = 0

		# language parser
		self.lang = language.langParse()

		# quiet output
		self.quiet = quiet

	def run(self):
		# connect to server
		try:
			self.reposit = harvestRepository(self.serverURI)
		except:
			self.finished = 1

		if not self.finished:
			# read in the languages
			self.lang.getLangDict()
			# start the harvest
			self.process()
		

	def finished(self):
		return self.finished

	def process(self):
		if self.finished == 1:
			return

		# progress indicator
		prog = progress()
		if self.quiet == 0:
			prog.begin()

		# get the records
		record = {}
		try:
			record = self.reposit.getRecord()
			while record is not None:
				if record.has_key('error') or record.has_key('deleted') or record.has_key('uncaught'):
					# error record or deleted record, skip
					record = self.reposit.getRecord()
					continue
				try:
					# hIdentifier is not required to be cross repository
					# unique (although it usually is), so prepend url
					record['recHash'] = make_hash(record['iOAIURL'] + record['hIdentifier'])
				except:
					# hash error, could not generate
					swrite ('H')
					hErr('HashError: Could not generate. >> %s' % self.serverURI)
					record = self.reposit.getRecord()
					continue
				self.addRecord(record)

				# progress indicator
				prog.dot()
				# get next record
				record = self.reposit.getRecord()

		except:
			swrite('L')
			hErr("ListError: Error in the record list: " + str(sys.exc_info()[0]))
			self.finished = 1
			return

		if self.quiet == 0:
			prog.done()
		self.finish()

	def addRecord(self,record):
		# sets for sql statement
		self.key_set = []
		self.val_set = []
		s = ''
		# filter all the strings for entry into mysql
		for k,v in record.iteritems():
			s = v
			if isList(v):
				s = u'\n'.join(v)
			self.key_set.append(k)
			self.val_set.append(s)

		# process language identification
		self.key_set.append('cLanguage')
		self.val_set.append(self.lang.parseLang(record['mLanguage']))

		# check for url
		self.key_set.append('cValidUrl')
		self.val_set.append(checkUrl(record))

		# merge into the insert dict
		insertdict = lists2dict(self.key_set, self.val_set)

		# insert the record
		dbinsertsafe('harvestSearch', insertdict, 'UPDATE cDups = cDups + 1');


	def finish(self):
		self.finished = 1
		self.complete = 1

def main():
	threadList = []
	threaded = 1
	maxThreads = g['MAXTHR']

	print "Starting Harvest"

	# current repository in line
	repPos = 0

	# total number of repositories
	maxRep = len(repositories)

	# number of active threads
	thrdCnt = len(threadList)

	# loop until finished
	while repPos < maxRep:

		# as long as there are free threads and more repositories
		while thrdCnt < maxThreads and repPos < maxRep:
			# add another thread
			rThr = repoHarv(repositories[repPos], threaded)
			repPos += 1

			if threaded == 0:
				print "Connecting to " + repositories[repPos - 1]  + " ..."
				rThr.process()

			else:
				threadList.append(rThr)
				rThr.start()
				swrite( 'S(%s/%s)' % (repPos,maxRep) )

			# how many threads are out there
			thrdCnt = len(threadList)

		# while there are more repositories and there are threads active
		while repPos < maxRep and thrdCnt > 0:
			# close the finished threads
			for t in threadList:
				if t.finished == 1 or not t.isAlive():
					t.join()

					# did it complete?
					if t.complete == 0:
						hErr('FailedOrIncompleteHarvest: %s' % t.serverURI)
					else:
						logDone(t.serverURI)

					threadList.remove(t)

					if t.finished != 1: # dead thread
						swrite( 'X' )
					else: # thread finished (or error was caught)
						swrite( 'F' )
					del t

			thrdCnt = len(threadList)
			# start up addition threads if needed
			if thrdCnt < maxThreads:
				rThr = repoHarv(repositories[repPos], threaded)
				repPos += 1
				threadList.append(rThr)
				rThr.start()
				swrite( 'N(%s/%s)' % (repPos,maxRep) )
				thrdCnt = len(threadList)

			# sleep main
			sleep(1)

	# if there are no more repositories
	if repPos >= maxRep:
		for t in threadList:
			t.join()
			swrite( 'D' )

	print
	print "Harvest Complete."
	

# file is being executed, start main
if __name__ == '__main__':
	override = ' '.join(sys.argv[1:])
	if override == 'reset':
		confirm = raw_input("Delete all existing data and recreate tables? (type yes to confirm)")
		if confirm == 'yes':
			print "Reseting table..."
			recreateTable()

		sys.exit(0)

	elif len(override) > 0:
		repositories = read_reposit(override)
	main()

# eof

